<!DOCTYPE html>



  


<html class="theme-next mist use-motion" lang="zh-CN">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta name="theme-color" content="#222">
<meta name="baidu-site-verification" content="code-Vc9hmj7JE9" />
<meta name="google-site-verification" content="tMVEtTlHbEdIEga44DZi47Yu8Pl2shFKVZcqz6rfSW0" />
<script>
    (function () {
        if ('') {
            if (prompt('请输入文章密码') !== '') {
                alert('密码错误！');
                if (history.length === 1) {
                    location.replace("https://kun-bin.github.io/");
                } else {
                    history.back();
                }
            }
        }
    })();
</script>








<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />
















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />







<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.4" rel="stylesheet" type="text/css" />
<link rel="stylesheet" type="text/css" href="/css/matery.css">

  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=5.1.4">


  <link rel="mask-icon" href="/images/logo.svg?v=5.1.4" color="#222">





  <meta name="keywords" content="crowdsourcing,Combinatorial Multi Armed Bandit," />










<meta name="description" content="今天的我真的超级难过了。但是再难过也没用，看看论文好了，希望等月底能好点。 啊月底还是他生日，没能坚持到月底真是太难过了。">
<meta property="og:type" content="article">
<meta property="og:title" content="Combinatorial Multi-Armed Bandit Based Unknown Worker Recruitment in Heterogeneous Crowdsensing">
<meta property="og:url" content="https://likun1208.github.io/2020/07/05/%E8%AE%BA%E6%96%87%E8%AE%B0%E5%BD%95-Combinatorial%20Multi-Armed%20Bandit%20Based%20Unknown%20Worker%20Recruitment%20in%20Heterogeneous%20Crowdsensing/index.html">
<meta property="og:site_name" content="左边">
<meta property="og:description" content="今天的我真的超级难过了。但是再难过也没用，看看论文好了，希望等月底能好点。 啊月底还是他生日，没能坚持到月底真是太难过了。">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2020-07-05T06:31:25.000Z">
<meta property="article:modified_time" content="2020-12-21T06:31:24.705Z">
<meta property="article:author" content="Kun Li">
<meta property="article:tag" content="crowdsourcing">
<meta property="article:tag" content="Combinatorial Multi Armed Bandit">
<meta name="twitter:card" content="summary">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Mist',
    version: '5.1.4',
    sidebar: {"position":"left","display":"always","offset":12,"b2t":false,"scrollpercent":true,"onmobile":false},
    fancybox: true,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    duoshuo: {
      userId: '0',
      author: 'Author'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="https://likun1208.github.io/2020/07/05/论文记录-Combinatorial Multi-Armed Bandit Based Unknown Worker Recruitment in Heterogeneous Crowdsensing/"/>





  <title>Combinatorial Multi-Armed Bandit Based Unknown Worker Recruitment in Heterogeneous Crowdsensing | 左边</title>
  








<meta name="generator" content="Hexo 4.2.0"></head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-CN">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">左边</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle"></p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            Home
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            Tags
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            Categories
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            Archives
          </a>
        </li>
      

      
        <li class="menu-item menu-item-search">
          
            <a href="javascript:;" class="popup-trigger">
          
            
              <i class="menu-item-icon fa fa-search fa-fw"></i> <br />
            
            Search
          </a>
        </li>
      
    </ul>
  

  
    <div class="site-search">
      
  <div class="popup search-popup local-search-popup">
  <div class="local-search-header clearfix">
    <span class="search-icon">
      <i class="fa fa-search"></i>
    </span>
    <span class="popup-btn-close">
      <i class="fa fa-times-circle"></i>
    </span>
    <div class="local-search-input-wrapper">
      <input autocomplete="off"
             placeholder="Searching..." spellcheck="false"
             type="text" id="local-search-input">
    </div>
  </div>
  <div id="local-search-result"></div>
</div>



    </div>
  
</nav>



 </div>
    </header>
	<script type="text/javascript" src="/js/echarts.min.js"></script>
    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="https://likun1208.github.io/2020/07/05/%E8%AE%BA%E6%96%87%E8%AE%B0%E5%BD%95-Combinatorial%20Multi-Armed%20Bandit%20Based%20Unknown%20Worker%20Recruitment%20in%20Heterogeneous%20Crowdsensing/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="Kun Li">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/header.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="左边">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">Combinatorial Multi-Armed Bandit Based Unknown Worker Recruitment in Heterogeneous Crowdsensing</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">Posted on</span>
              
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2020-07-05T14:31:25+08:00">
                2020-07-05
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">In</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/%E8%AE%BA%E6%96%87/" itemprop="url" rel="index">
                    <span itemprop="name">论文</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
          

          
          

          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <p>今天的我真的超级难过了。但是再难过也没用，看看论文好了，希望等月底能好点。</p>
<p>啊月底还是他生日，没能坚持到月底真是太难过了。</p>
<a id="more"></a>
<h2 id="Introduction"><a href="#Introduction" class="headerlink" title="Introduction"></a>Introduction</h2><ol>
<li>本文研究问题：异构群智感知系统（也就是众包）中对未知worker的招募</li>
<li>本文场景：requester招募workers收集某城市交通路口一段时间内的交通数据，整个收集过程分为多轮，每轮包含一些和地点有关的任务，对应一个交通路口。每个任务有权重，表示重要性。每个worker能做一个或多个任务，且不同worker能做的任务可能不一样。worker会告诉平台自己能做的任务和期望收到的费用。worker完成任务的质量服从未知分布。</li>
<li>本文目的：设计worker招聘方案，在给定预算的情况下，最大限度提高总任务完成质量。</li>
<li>本文面临的挑战：平台不知道worker的质量分布</li>
<li>本文解决挑战的方法：让worker先完成一些任务，然后从任务结果里学习worker的质量，最后从中找最好的worker，简单来说分为exploration和exploitation。本文需要平衡这两个过程，从而实现目标（这么看一开始被完成的那些任务就被牺牲了）。</li>
<li>本文将上述问题概括为组合多臂老虎机模型（ Combinatorial Multi Armed Bandit），并且说和现存的CMAB模型都不一样；然后本文用扩展的上置信界算法（Upper Confidence Bound）。多臂老虎机模型我之前听说过，但是完全不了解，所以要先查一下。</li>
<li>本文贡献：<ol>
<li>介绍了这个场景并把它概括为多臂老虎机</li>
<li>用UCB来解决这个问题</li>
<li>研究了扩展问题：worker质量和期望收费都不知道的场景</li>
<li>做了仿真实验，分析了性能</li>
</ol>
</li>
</ol>
<h2 id="Combinatorial-Multi-Armed-Bandit"><a href="#Combinatorial-Multi-Armed-Bandit" class="headerlink" title="Combinatorial Multi Armed Bandit"></a>Combinatorial Multi Armed Bandit</h2><ol>
<li>实质是未知概率情况下的选择问题，比如赌博</li>
<li>具体来说，重复一个选择过程，每次有k个选项或动作可供选择，每次选择一个动作后会获得相应的奖励。目标是为了最大化k次后的奖励。选项对应的收益服从某种未知概率分布，对于实验者本人而言是黑箱，因此需要采取各种可能的方式来最大化收益。</li>
<li>基础思路：每一轮根据之前的结果更新对收益的期望，期望计算方法为[之前采取该选项所得到的所有收益]/[之前采取过该选项的次数]，也就是平均每次得到的收益；只要时间够长，这个算出来的期望就会接近真实收益。</li>
</ol>
<h2 id="System-model-and-problem"><a href="#System-model-and-problem" class="headerlink" title="System model and problem"></a>System model and problem</h2><ol>
<li><p>字母符号表示：</p>
<p>|               名称               |                             含义                             |<br>| :———————————————: | :—————————————————————————————: |<br>|                t                 |                       当前轮数，第t轮                        |<br>|                N                 |                N个workers的集合，第i个worker                 |<br>|                M                 |                   M个任务的集合，第j个任务                   |<br>|                B                 |                             预算                             |<br>|              $w_j$               |            第j个任务的权重，所有权重加起来的和是1            |<br>|                L                 |              每个worker会向平台提交L个任务候选               |<br>|      $p_i^l=<M_i^l,c_i^l>$       | 第i个worker提交的第l个选项，其中$M_i^l$表示该worker的任务候选集合，$c_i^l$表示收费 |<br>| $c_i^l=\varepsilon_i f(|M_i^l|)$ | $\varepsilon_i$是收费参数，不同worker不一样，比如有的worker设备特别棒，这个参数就会大一点，参数是已知公开（先验）的，本文也考虑了未知的情况；$f()$是单调增函数，意思是任务完成越多收益越大，是已知公开的，且大家都一样；c的取值被规范化到0和1之间 |<br>|     $P_i=\{p_i^l|1&lt;=l&lt;=L\}$      |                  第i个worker提交的选项集合                   |<br>|        $P=U_{i\in N}P_i $        |                         所有选项集合                         |<br>|      $q_{i,j}^t|j\in M_i^l$      | 非负随机量，范围在0和1之间，表示第i个worker在第t轮完成第j个任务的任务质量，服从某种未知分布，期望是$q_i$，期望也未知 |<br>|          $P^t\subset P$          |            第t轮中平台对所有workers所选的选项集合            |<br>|          $p_i^l\in P^t$          |            第t轮平台对第i个worker选了其第l个选项             |<br>|            $u^j(P^t)$            | 第t轮采用方案$P^t$时的第j个任务的最终质量（所有完成任务结果中最好的那个） |<br>|             $u(P^t)$             | 第t轮采用方案$P^t$时所有任务的最终质量和，也就是上一个符号乘权重再加起来 |<br>|            $n_i^l(t)$            |               第i个worker的第l个选项被选的次数               |<br>|             $n_i(t)$             |                  第i个worker被学习过的次数                   |<br>|       $\overline{q}_i(t)$        |             截至到第t轮学习到的第i个worker的质量             |</p>
<p>需要注意：虽然worker可以提交L个任务候选，但是每一轮只能最终完成一个选项，这里假设$c_i^1$到$c_i^L$是从小到大排的，也就是说最后一个的收费最高，且实际中，c的取值一般和M的长度（就是任务数量）正相关。</p>
<ul>
<li>这里有个奇怪的问题，我以为每个选项就是单独一个任务，然后c是对应的收费，但是看起来每个选项是任务集合，然后c是收费，也就是说比如有5个任务用abcde表示，某个worker的选项就会是{a,b,收费3}{b,c,d,收费5}{a,c,d,e,收费10}，这样看起来好奇怪。希望后面有解释。</li>
<li>虽然这样的设定有点别扭，不过解释是说：每一轮每个worker完成$|M_i^l|$个任务，也就会学习到到$|M_i^l|$个任务质量，就是说任务质量会被学习$|M_i^l|$次，这和传统CMAB不一样。</li>
<li>每轮每个worker最多定一个选项（也可以不选）</li>
</ul>
</li>
<li><p>要研究的问题：给定预算，每轮招募K个workers，使得所有轮中完成的所有任务的权重加起来最大。</p>
</li>
<li><p>数学模型：</p>
<p>目标函数最大化：$E[\sum_{t\geq1}u(P^t) ]$    所有轮下来总期望收益最大</p>
<p>约束：$\sum_{t\geq1}\sum_{p_i^l\in P^t}c_i^l\leq B$    花费不超过预算</p>
<p>​            $|P^t|=K \  for\ \forall t&gt;1$    每一轮都招K个workers，不多不少</p>
<p>​            $\sum_{l=1}^LI\{p_i^l\in P^t\}\leq 1$    每个worker的选项最多一个</p>
</li>
</ol>
<h2 id="Algorithm-Design"><a href="#Algorithm-Design" class="headerlink" title="Algorithm Design"></a>Algorithm Design</h2><ol>
<li><p>本文模型：K臂的组合多臂赌博机</p>
</li>
<li><p>本文方法：</p>
<ol>
<li>扩展的上置信界算法（UCB）学习任务质量</li>
<li>增加了对最大化权重的考虑</li>
<li>每轮用贪心算法招K个workers：最大化任务质量和招募费用的比（单位费用的任务质量最大化）</li>
</ol>
<h3 id="原本的UCB算法"><a href="#原本的UCB算法" class="headerlink" title="原本的UCB算法"></a>原本的UCB算法</h3><ol>
<li>总的来说就是估计置信区间</li>
<li>我们认为真实的那个未知概率或者说收益是p，而根据尝试和计算推断出的概率是$\widetilde{p}$，这两个概率之间存在差值，即：$\widetilde{p}-\Delta \leq p \leq \widetilde{p}+\Delta$，这个范围就是置信区间，算法的目的就是通过一次次尝试缩小置信区间</li>
<li>该算法的流程是在所有臂里找$\widetilde{p}+\Delta$最大的那个，根据一系列完全没看的数学定理，$\Delta=\sqrt{2\ln T /n}$，T是目前进行过的轮数，n是这个臂已经被选过的次数，每一轮执行完会更新数据。具体来说，$\widetilde{p}$最大，选这个选项的收益就越大，而$\Delta$越大，这个选项之前被选中的次数就越小。</li>
<li>总结一下就是会考虑每个臂已经估计过的历史记录，尽可能去探索次数较少和收益较高的臂，兼顾收益和探索。</li>
</ol>
<h3 id="本文的算法"><a href="#本文的算法" class="headerlink" title="本文的算法"></a>本文的算法</h3><ol>
<li>在第t轮中，若第i个worker的第l个选项被选中，则$n_i^l(t)=n_i^l(t-1)+1$（就是比上一轮的多1），反之则保持上一轮的值不变</li>
<li>$n_i(t)$的值是第t轮时的第i个worker每个选项的$n_i^l(t)$和该选项任务数（也就是$|M_i^l|$）相乘，然后所有的加起来，表示第i个worker的质量被学习过的次数</li>
<li>用普通的总值/总次数更新worker的质量（$\overline {q}_i(t)$），用不太一样的UCB平衡探索和收益（$\widehat{q}_i (t)$）</li>
<li>每一轮都是最大化权重*$\widehat{q}_i (t)$，也就是根据之前结果的信息推断出的最大收益</li>
</ol>
<h3 id="本文的流程"><a href="#本文的流程" class="headerlink" title="本文的流程"></a>本文的流程</h3><ol>
<li>最一开始，对于每个worker平台都让他去完成候选列表中的第一个任务（就是最便宜的那个）,由此初始化$n_i^l(t)$、$n_i(t)$、$\overline{q}_i(t)$。</li>
<li>接下来的每一轮中，都以最大化单位费用的收益增长为目的来选择K个worker和它们的任务，也就是说([选择这个任务选项的收益]-[选之前的收益])/[选这个任务的开销]，要找使得这个式子最大的那个任务选项。要注意这一步中，当某个worker已经被选了任务，那他的其他选项都不会再被考虑</li>
<li>第t轮的K个worker选好以后，开始各自完成任务，做完以后平台计算任务质量，由此更新$n_i^l(t)$、$n_i(t)$、$\overline{q}_i(t)$、$\widehat{q}_i (t)$。同时，目前为止的所有轮获得的收益也更新了，平台根据预算还剩多少决定是否进行下一轮。</li>
</ol>
<h3 id="算法性能分析"><a href="#算法性能分析" class="headerlink" title="算法性能分析"></a>算法性能分析</h3><ol>
<li>实质是01背包问题</li>
<li>经过一系列我还没看的计算，该算法复杂度是$O(NLK^3\ln \tau(B))$</li>
</ol>
</li>
</ol>
<h2 id="扩展"><a href="#扩展" class="headerlink" title="扩展"></a>扩展</h2><ol>
<li><p>扩展问题场景：所有worker的质量和收费都未知，收费未知是指$c_i^l=\varepsilon_i f(|M_i^l|)$这个公式里的参数$\varepsilon_i$未知，公式里的函数$f()$是公开的。具体来说，在第t轮，worker的任务已经选定后，worker根据当前电量、环境、网络等估计一个第t轮的收费参数$\varepsilon_i^t$，该值在0和1之间，且有下限$\varepsilon_{min}$，所有轮的$\varepsilon_i^t$独立同分布，分布未知，期望是$\varepsilon_i$。</p>
</li>
<li><p>每一轮开始时，首先是平台选worker和任务，然后是worker报价，接着平台算一下预算够不够，不够的话就结束，反之就进入做任务的环节，之后的流程和上一部分一样。</p>
</li>
<li><p>问题在于：$\varepsilon_i$也需要学习，而且每一轮会被学习一次，这和之前的任务质量不太一样。</p>
<h3 id="本文方法"><a href="#本文方法" class="headerlink" title="本文方法"></a>本文方法</h3><ol>
<li>新增一个符号表示：$m_i(t)=\sum_{l=1}^Ln_i^l(t)$，表示$\varepsilon_i$目前被学习过的次数（第i个worker的所有选项目前被选过的次数）</li>
<li>新增另一个符号表示：$\overline \varepsilon_i (t) $，计算方法和前面p那个类似，也是[在此之前的值*在此之前的次数+这次的值]/[在此之前的次数+1]</li>
<li>同样也新增了$\widehat  \varepsilon_i (t)$，和前面的一样</li>
<li>把之前那个目标函数里的费用部分用这里新的符号改写然后化简，但是这里化简以后的没看懂（问了一下作者，是从regret部分分析出来的，然后又看了看之前没看的证明，发现是证明部分分析的）</li>
<li>于是整个流程就和之前的一模一样，只是目标函数换了</li>
<li>算法性能分析和前面一样，还没看，感觉不重要</li>
</ol>
</li>
</ol>
<h2 id="Performance-Evaluation"><a href="#Performance-Evaluation" class="headerlink" title="Performance Evaluation"></a>Performance Evaluation</h2><ol>
<li>实验部分对平台的介绍格外简单，用的公开数据集，这部分没什么能说的</li>
<li>实验主要关注：期望质量和期望费用（就是前面计算的俩参数）</li>
<li>实验内容是和另一种常用的CMAB的算法做对比</li>
<li>针对第一个算法：分析了预算的影响（500-1000），招募工人数K的变化（得出K小一些更好，但是意味着要来更多轮），用均匀分布作为例子对比了准确率</li>
<li>针对第二个算法：估计了质量和预算的关系，改变工人数之后的性能（和上一个不太一样了）</li>
</ol>
<h2 id="Conclusion"><a href="#Conclusion" class="headerlink" title="Conclusion"></a>Conclusion</h2><ol>
<li>没啥总结的，这个论文就这样了</li>
<li>之后有时间就看看性能证明那里，不过个人觉得十有八九是已有证明改编的</li>
<li>看了一点证明，还没完全看懂，大致了解思路了，不过不打算继续看了……</li>
</ol>

      
    </div>
    
    
    

    

    

    
	
	<div>
		
		<div>
    
        <div style="text-align:center;color: #ccc;font-size:14px;">-------------本文结束<i class="fa fa-paw"></i>感谢您的阅读-------------</div>
    
</div>
		
	</div>
	
    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/crowdsourcing/" rel="tag"><i class="fa fa-tag"></i> crowdsourcing</a>
          
            <a href="/tags/Combinatorial-Multi-Armed-Bandit/" rel="tag"><i class="fa fa-tag"></i> Combinatorial Multi Armed Bandit</a>
          
        </div>
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2020/07/05/%E8%AE%BA%E6%96%87%E8%AE%B0%E5%BD%95-%E9%87%8D%E6%9E%84%E6%AF%94%E7%89%B9%E5%B8%81%E8%B4%B9%E7%8E%87%E5%B8%82%E5%9C%BA/" rel="next" title="论文记录-重构比特币费率市场">
                <i class="fa fa-chevron-left"></i> 论文记录-重构比特币费率市场
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2020/07/06/blender%E5%AD%A6%E4%B9%A0%E8%AE%B0%E5%BD%95-1/" rel="prev" title="blender学习记录-1">
                blender学习记录-1 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          


          

  



        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            Table of Contents
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            Overview
          </li>
        </ul>
      

      <section class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
            
              <img class="site-author-image" itemprop="image"
                src="/images/header.jpg"
                alt="Kun Li" />
            
              <p class="site-author-name" itemprop="name">Kun Li</p>
              <p class="site-description motion-element" itemprop="description">三月樱，六月雪，也望尘莫及</p>
          </div>

          <nav class="site-state motion-element">

            
              <div class="site-state-item site-state-posts">
              
				<a href="/archives/">
              
                  <span class="site-state-item-count">54</span>
                  <span class="site-state-item-name">posts</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-categories">
                <a href="/categories/index.html">
                  <span class="site-state-item-count">9</span>
                  <span class="site-state-item-name">categories</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-tags">
                <a href="/tags/index.html">
                  <span class="site-state-item-count">27</span>
                  <span class="site-state-item-name">tags</span>
                </a>
              </div>
            

          </nav>

          

          
            <div class="links-of-author motion-element">
                
                  <span class="links-of-author-item">
                    <a href="https://github.com/likun1208" target="_blank" title="GitHub">
                      
                        <i class="fa fa-fw fa-github"></i>GitHub</a>
                  </span>
                
                  <span class="links-of-author-item">
                    <a href="likun@mail.bnu.edu.cn" target="_blank" title="E-Mail">
                      
                        <i class="fa fa-fw fa-envelope"></i>E-Mail</a>
                  </span>
                
            </div>
          

          
          

          
          

          

        </div>
      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#Introduction"><span class="nav-number">1.</span> <span class="nav-text">Introduction</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Combinatorial-Multi-Armed-Bandit"><span class="nav-number">2.</span> <span class="nav-text">Combinatorial Multi Armed Bandit</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#System-model-and-problem"><span class="nav-number">3.</span> <span class="nav-text">System model and problem</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Algorithm-Design"><span class="nav-number">4.</span> <span class="nav-text">Algorithm Design</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#原本的UCB算法"><span class="nav-number">4.1.</span> <span class="nav-text">原本的UCB算法</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#本文的算法"><span class="nav-number">4.2.</span> <span class="nav-text">本文的算法</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#本文的流程"><span class="nav-number">4.3.</span> <span class="nav-text">本文的流程</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#算法性能分析"><span class="nav-number">4.4.</span> <span class="nav-text">算法性能分析</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#扩展"><span class="nav-number">5.</span> <span class="nav-text">扩展</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#本文方法"><span class="nav-number">5.1.</span> <span class="nav-text">本文方法</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Performance-Evaluation"><span class="nav-number">6.</span> <span class="nav-text">Performance Evaluation</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Conclusion"><span class="nav-number">7.</span> <span class="nav-text">Conclusion</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Kun Li</span>

  
</div>


  <div class="powered-by">Powered by <a class="theme-link" target="_blank" href="https://hexo.io">Hexo</a></div>



  <span class="post-meta-divider">|</span>



  <div class="theme-info">Theme &mdash; <a class="theme-link" target="_blank" href="https://github.com/iissnan/hexo-theme-next">NexT.Mist</a> v5.1.4</div>




        







        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
          <span id="scrollpercent"><span>0</span>%</span>
        
      </div>
    

    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  












  
  
    <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>
  

  
  
    <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>
  

  
  
    <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>
  


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.4"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.4"></script>



  
  

  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.4"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.4"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.4"></script>



  


  




	





  





  












  

  <script type="text/javascript">
    // Popup Window;
    var isfetched = false;
    var isXml = true;
    // Search DB path;
    var search_path = "search.xml";
    if (search_path.length === 0) {
      search_path = "search.xml";
    } else if (/json$/i.test(search_path)) {
      isXml = false;
    }
    var path = "/" + search_path;
    // monitor main search box;

    var onPopupClose = function (e) {
      $('.popup').hide();
      $('#local-search-input').val('');
      $('.search-result-list').remove();
      $('#no-result').remove();
      $(".local-search-pop-overlay").remove();
      $('body').css('overflow', '');
    }

    function proceedsearch() {
      $("body")
        .append('<div class="search-popup-overlay local-search-pop-overlay"></div>')
        .css('overflow', 'hidden');
      $('.search-popup-overlay').click(onPopupClose);
      $('.popup').toggle();
      var $localSearchInput = $('#local-search-input');
      $localSearchInput.attr("autocapitalize", "none");
      $localSearchInput.attr("autocorrect", "off");
      $localSearchInput.focus();
    }

    // search function;
    var searchFunc = function(path, search_id, content_id) {
      'use strict';

      // start loading animation
      $("body")
        .append('<div class="search-popup-overlay local-search-pop-overlay">' +
          '<div id="search-loading-icon">' +
          '<i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>' +
          '</div>' +
          '</div>')
        .css('overflow', 'hidden');
      $("#search-loading-icon").css('margin', '20% auto 0 auto').css('text-align', 'center');

      $.ajax({
        url: path,
        dataType: isXml ? "xml" : "json",
        async: true,
        success: function(res) {
          // get the contents from search data
          isfetched = true;
          $('.popup').detach().appendTo('.header-inner');
          var datas = isXml ? $("entry", res).map(function() {
            return {
              title: $("title", this).text(),
              content: $("content",this).text(),
              url: $("url" , this).text()
            };
          }).get() : res;
          var input = document.getElementById(search_id);
          var resultContent = document.getElementById(content_id);
          var inputEventFunction = function() {
            var searchText = input.value.trim().toLowerCase();
            var keywords = searchText.split(/[\s\-]+/);
            if (keywords.length > 1) {
              keywords.push(searchText);
            }
            var resultItems = [];
            if (searchText.length > 0) {
              // perform local searching
              datas.forEach(function(data) {
                var isMatch = false;
                var hitCount = 0;
                var searchTextCount = 0;
                var title = data.title.trim();
                var titleInLowerCase = title.toLowerCase();
                var content = data.content.trim().replace(/<[^>]+>/g,"");
                var contentInLowerCase = content.toLowerCase();
                var articleUrl = decodeURIComponent(data.url);
                var indexOfTitle = [];
                var indexOfContent = [];
                // only match articles with not empty titles
                if(title != '') {
                  keywords.forEach(function(keyword) {
                    function getIndexByWord(word, text, caseSensitive) {
                      var wordLen = word.length;
                      if (wordLen === 0) {
                        return [];
                      }
                      var startPosition = 0, position = [], index = [];
                      if (!caseSensitive) {
                        text = text.toLowerCase();
                        word = word.toLowerCase();
                      }
                      while ((position = text.indexOf(word, startPosition)) > -1) {
                        index.push({position: position, word: word});
                        startPosition = position + wordLen;
                      }
                      return index;
                    }

                    indexOfTitle = indexOfTitle.concat(getIndexByWord(keyword, titleInLowerCase, false));
                    indexOfContent = indexOfContent.concat(getIndexByWord(keyword, contentInLowerCase, false));
                  });
                  if (indexOfTitle.length > 0 || indexOfContent.length > 0) {
                    isMatch = true;
                    hitCount = indexOfTitle.length + indexOfContent.length;
                  }
                }

                // show search results

                if (isMatch) {
                  // sort index by position of keyword

                  [indexOfTitle, indexOfContent].forEach(function (index) {
                    index.sort(function (itemLeft, itemRight) {
                      if (itemRight.position !== itemLeft.position) {
                        return itemRight.position - itemLeft.position;
                      } else {
                        return itemLeft.word.length - itemRight.word.length;
                      }
                    });
                  });

                  // merge hits into slices

                  function mergeIntoSlice(text, start, end, index) {
                    var item = index[index.length - 1];
                    var position = item.position;
                    var word = item.word;
                    var hits = [];
                    var searchTextCountInSlice = 0;
                    while (position + word.length <= end && index.length != 0) {
                      if (word === searchText) {
                        searchTextCountInSlice++;
                      }
                      hits.push({position: position, length: word.length});
                      var wordEnd = position + word.length;

                      // move to next position of hit

                      index.pop();
                      while (index.length != 0) {
                        item = index[index.length - 1];
                        position = item.position;
                        word = item.word;
                        if (wordEnd > position) {
                          index.pop();
                        } else {
                          break;
                        }
                      }
                    }
                    searchTextCount += searchTextCountInSlice;
                    return {
                      hits: hits,
                      start: start,
                      end: end,
                      searchTextCount: searchTextCountInSlice
                    };
                  }

                  var slicesOfTitle = [];
                  if (indexOfTitle.length != 0) {
                    slicesOfTitle.push(mergeIntoSlice(title, 0, title.length, indexOfTitle));
                  }

                  var slicesOfContent = [];
                  while (indexOfContent.length != 0) {
                    var item = indexOfContent[indexOfContent.length - 1];
                    var position = item.position;
                    var word = item.word;
                    // cut out 100 characters
                    var start = position - 20;
                    var end = position + 80;
                    if(start < 0){
                      start = 0;
                    }
                    if (end < position + word.length) {
                      end = position + word.length;
                    }
                    if(end > content.length){
                      end = content.length;
                    }
                    slicesOfContent.push(mergeIntoSlice(content, start, end, indexOfContent));
                  }

                  // sort slices in content by search text's count and hits' count

                  slicesOfContent.sort(function (sliceLeft, sliceRight) {
                    if (sliceLeft.searchTextCount !== sliceRight.searchTextCount) {
                      return sliceRight.searchTextCount - sliceLeft.searchTextCount;
                    } else if (sliceLeft.hits.length !== sliceRight.hits.length) {
                      return sliceRight.hits.length - sliceLeft.hits.length;
                    } else {
                      return sliceLeft.start - sliceRight.start;
                    }
                  });

                  // select top N slices in content

                  var upperBound = parseInt('1');
                  if (upperBound >= 0) {
                    slicesOfContent = slicesOfContent.slice(0, upperBound);
                  }

                  // highlight title and content

                  function highlightKeyword(text, slice) {
                    var result = '';
                    var prevEnd = slice.start;
                    slice.hits.forEach(function (hit) {
                      result += text.substring(prevEnd, hit.position);
                      var end = hit.position + hit.length;
                      result += '<b class="search-keyword">' + text.substring(hit.position, end) + '</b>';
                      prevEnd = end;
                    });
                    result += text.substring(prevEnd, slice.end);
                    return result;
                  }

                  var resultItem = '';

                  if (slicesOfTitle.length != 0) {
                    resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + highlightKeyword(title, slicesOfTitle[0]) + "</a>";
                  } else {
                    resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + title + "</a>";
                  }

                  slicesOfContent.forEach(function (slice) {
                    resultItem += "<a href='" + articleUrl + "'>" +
                      "<p class=\"search-result\">" + highlightKeyword(content, slice) +
                      "...</p>" + "</a>";
                  });

                  resultItem += "</li>";
                  resultItems.push({
                    item: resultItem,
                    searchTextCount: searchTextCount,
                    hitCount: hitCount,
                    id: resultItems.length
                  });
                }
              })
            };
            if (keywords.length === 1 && keywords[0] === "") {
              resultContent.innerHTML = '<div id="no-result"><i class="fa fa-search fa-5x" /></div>'
            } else if (resultItems.length === 0) {
              resultContent.innerHTML = '<div id="no-result"><i class="fa fa-frown-o fa-5x" /></div>'
            } else {
              resultItems.sort(function (resultLeft, resultRight) {
                if (resultLeft.searchTextCount !== resultRight.searchTextCount) {
                  return resultRight.searchTextCount - resultLeft.searchTextCount;
                } else if (resultLeft.hitCount !== resultRight.hitCount) {
                  return resultRight.hitCount - resultLeft.hitCount;
                } else {
                  return resultRight.id - resultLeft.id;
                }
              });
              var searchResultList = '<ul class=\"search-result-list\">';
              resultItems.forEach(function (result) {
                searchResultList += result.item;
              })
              searchResultList += "</ul>";
              resultContent.innerHTML = searchResultList;
            }
          }

          if ('auto' === 'auto') {
            input.addEventListener('input', inputEventFunction);
          } else {
            $('.search-icon').click(inputEventFunction);
            input.addEventListener('keypress', function (event) {
              if (event.keyCode === 13) {
                inputEventFunction();
              }
            });
          }

          // remove loading animation
          $(".local-search-pop-overlay").remove();
          $('body').css('overflow', '');

          proceedsearch();
        }
      });
    }

    // handle and trigger popup window;
    $('.popup-trigger').click(function(e) {
      e.stopPropagation();
      if (isfetched === false) {
        searchFunc(path, 'local-search-input', 'local-search-result');
      } else {
        proceedsearch();
      };
    });

    $('.popup-btn-close').click(onPopupClose);
    $('.popup').click(function(e){
      e.stopPropagation();
    });
    $(document).on('keyup', function (event) {
      var shouldDismissSearchPopup = event.which === 27 &&
        $('.search-popup').is(':visible');
      if (shouldDismissSearchPopup) {
        onPopupClose();
      }
    });
  </script>





  

  

  

  
  

  
  
    <script type="text/x-mathjax-config">
      MathJax.Hub.Config({
        tex2jax: {
          inlineMath: [ ['$','$'], ["\\(","\\)"]  ],
          processEscapes: true,
          skipTags: ['script', 'noscript', 'style', 'textarea', 'pre', 'code']
        }
      });
    </script>

    <script type="text/x-mathjax-config">
      MathJax.Hub.Queue(function() {
        var all = MathJax.Hub.getAllJax(), i;
        for (i=0; i < all.length; i += 1) {
          all[i].SourceElement().parentNode.className += ' has-jax';
        }
      });
    </script>
    <script type="text/javascript" src="//cdn.bootcss.com/mathjax/2.7.1/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
  


  

  

  
  <style>
    .copy-btn {
      display: inline-block;
      padding: 6px 12px;
      font-size: 13px;
      font-weight: 700;
      line-height: 20px;
      color: #333;
      white-space: nowrap;
      vertical-align: middle;
      cursor: pointer;
      background-color: #eee;
      background-image: linear-gradient(#fcfcfc, #eee);
      border: 1px solid #d5d5d5;
      border-radius: 3px;
      user-select: none;
      outline: 0;
    }

    .highlight-wrap .copy-btn {
      transition: opacity .3s ease-in-out;
      opacity: 0;
      padding: 2px 6px;
      position: absolute;
      right: 4px;
      top: 8px;
    }

    .highlight-wrap:hover .copy-btn,
    .highlight-wrap .copy-btn:focus {
      opacity: 1
    }

    .highlight-wrap {
      position: relative;
    }
  </style>
  
  <script>
    $('.highlight').each(function (i, e) {
      var $wrap = $('<div>').addClass('highlight-wrap')
      $(e).after($wrap)
      $wrap.append($('<button>').addClass('copy-btn').append('Copy').on('click', function (e) {
        var code = $(this).parent().find('.code').find('.line').map(function (i, e) {
          return $(e).text()
        }).toArray().join('\n')
        var ta = document.createElement('textarea')
        document.body.appendChild(ta)
        ta.style.position = 'absolute'
        ta.style.top = '0px'
        ta.style.left = '0px'
        ta.value = code
        ta.select()
        ta.focus()
        var result = document.execCommand('copy')
        document.body.removeChild(ta)
        
          if(result)$(this).text('Success')
          else $(this).text('Fail')
        
        $(this).blur()
      })).on('mouseleave', function (e) {
        var $b = $(this).find('.copy-btn')
        setTimeout(function () {
          $b.text('Copy')
        }, 300)
      }).append(e)
    })
  </script>

<script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"pluginRootPath":"live2dw/","pluginJsPath":"lib/","pluginModelPath":"assets/","tagMode":false,"model":{"jsonPath":"/live2dw/assets/z16.model.json"},"display":{"position":"left","width":175,"height":350},"mobile":{"show":false},"log":false});</script></body>
</html>
<!-- 页面点击小红心 -->
<script type="text/javascript" src="/js/src/clicklove.js"></script>
